"""
知乎热榜爬虫
"""

from typing import List, Dict, Any
import re
import json
import time
from bs4 import BeautifulSoup

from .base import BaseScraper
from ..core.config import get_settings
from ..core.logger import app_logger

settings = get_settings()


class ZhihuScraper(BaseScraper):
    """知乎热榜爬虫"""
    
    def __init__(self):
        super().__init__()
        # 知乎热榜页面URL
        self.hot_url = "https://www.zhihu.com/billboard"
        self.headers.update({
            'Referer': 'https://www.zhihu.com/',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Accept-Encoding': 'gzip, deflate, br',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1',
            'Cache-Control': 'no-cache',
            'Pragma': 'no-cache',
        })
    
    @property
    def name(self) -> str:
        return "知乎热榜"
    
    @property
    def description(self) -> str:
        return "抓取知乎热榜内容"
    
    async def scrape(self) -> List[Dict[str, Any]]:
        """抓取知乎热榜数据"""
        
        app_logger.info("开始抓取知乎热榜...")
        
        try:
            # 使用真实数据抓取
            trends = await self._scrape_zhihu_hot_real()
            
            app_logger.info(f"知乎热榜抓取完成，共 {len(trends)} 条")
            return trends
            
        except Exception as e:
            app_logger.error(f"知乎热榜抓取失败: {str(e)}")
            return []
    
    async def _scrape_zhihu_hot_real(self) -> List[Dict[str, Any]]:
        """抓取知乎热榜的真实实现"""
        try:
            # 获取页面内容
            html = await self.fetch(self.hot_url)
            if not html:
                raise Exception("获取知乎热榜页面失败")
            
            # 检查是否包含热榜数据
            if "403 Forbidden" in html or "403" in html:
                app_logger.warning("知乎返回403错误，可能触发反爬虫机制")
                return await self._get_mock_data()
            
            if "billboard" not in html and "热榜" not in html and "hot" not in html:
                app_logger.warning("知乎页面可能没有正确加载热榜数据")
                return await self._get_mock_data()
            
            # 解析热榜数据
            trends = await self._parse_trends_from_html(html)
            
            # 如果没有获取到数据，使用模拟数据
            if not trends:
                app_logger.warning("未获取到真实知乎热榜数据，使用模拟数据")
                return await self._get_mock_data()
            
            return trends
            
        except Exception as e:
            app_logger.error(f"抓取知乎热榜真实数据失败: {e}")
            # 出错时返回模拟数据
            return await self._get_mock_data()
    
    async def _parse_trends_from_html(self, html: str) -> List[Dict[str, Any]]:
        """从HTML中解析热榜数据"""
        trends = []
        
        try:
            # 检查是否有错误信息
            if "403" in html or "Forbidden" in html:
                app_logger.warning("知乎页面返回403错误")
                return []
            
            soup = BeautifulSoup(html, 'html.parser')
            
            # 查找热榜列表项 - 尝试多种选择器
            trend_items = (soup.select('.HotItem') or 
                          soup.select('[data-zop]') or 
                          soup.select('.HotList-item') or
                          soup.select('.List-item') or
                          soup.select('.Card a[href*="question"]'))
            
            app_logger.info(f"找到 {len(trend_items)} 个可能的热榜项")
            
            # 如果没有找到标准选择器，尝试查找包含热榜数据的script标签
            if not trend_items:
                script_tags = soup.find_all('script')
                for script in script_tags:
                    if script.string and ('hotList' in script.string or 'billboard' in script.string):
                        # 尝试提取JSON数据
                        try:
                            # 查找类似 hotList = [...] 或 billboard = [...] 的结构
                            json_matches = re.findall(r'(?:hotList|billboard)\s*=\s*(\[.*?\])\s*;', script.string, re.DOTALL)
                            for json_str in json_matches:
                                try:
                                    # 清理JSON字符串
                                    json_str = re.sub(r'(\w+):', r'"\1":', json_str)  # 为key添加引号
                                    data = json.loads(json_str)
                                    for i, item in enumerate(data[:20], 1):  # 取前20条
                                        target = item.get('target', {})
                                        title = target.get('title', '')
                                        if title:
                                            heat_score = item.get('detail_text', 0)
                                            # 如果热度为0，根据排名估算
                                            if heat_score == 0:
                                                heat_score = max(1, 21 - i) * 40000
                                                
                                            trends.append({
                                                "title": title,
                                                "description": target.get('excerpt', '') or target.get('description', ''),
                                                "rank": i,
                                                "heat_score": heat_score,
                                                "url": target.get('url', '') or f"https://www.zhihu.com/question/{target.get('id', '')}"
                                            })
                                    if trends:
                                        return trends
                                except Exception as e:
                                    app_logger.warning(f"解析知乎JSON数据失败: {e}")
                                    continue
                        except Exception as e:
                            app_logger.warning(f"查找知乎JSON数据失败: {e}")
            
            # 从HTML元素中解析数据
            for i, item in enumerate(trend_items[:20], 1):  # 取前20条
                try:
                    # 提取标题
                    title_elem = (item.select_one('.HotItem-title') or 
                                item.select_one('[data-zop] .title') or 
                                item.select_one('h2') or
                                item.select_one('.ContentItem-title') or
                                item.select_one('a'))
                    
                    title = ""
                    if not title_elem:
                        # 尝试从data-zop属性获取
                        zop_data = item.get('data-zop')
                        if zop_data:
                            try:
                                zop = json.loads(zop_data)
                                title = zop.get('title', '')
                            except:
                                title = item.get_text().strip().split('\n')[0]
                        else:
                            title = item.get_text().strip().split('\n')[0]
                    else:
                        title = title_elem.get_text().strip()
                    
                    if not title or len(title) < 2:
                        continue
                    
                    # 提取链接
                    link_elem = item.select_one('a') or item.find('a')
                    link = ""
                    if link_elem and link_elem.get('href'):
                        link = link_elem.get('href', '')
                        if link.startswith('//'):
                            link = 'https:' + link
                        elif link.startswith('/'):
                            link = 'https://www.zhihu.com' + link
                        elif not link.startswith('http'):
                            # 尝试从data-zop中获取
                            zop_data = item.get('data-zop')
                            if zop_data:
                                try:
                                    zop = json.loads(zop_data)
                                    question_id = zop.get('id', '')
                                    if question_id:
                                        link = f"https://www.zhihu.com/question/{question_id}"
                                except:
                                    pass
                    else:
                        # 从data-zop中获取链接
                        zop_data = item.get('data-zop')
                        if zop_data:
                            try:
                                zop = json.loads(zop_data)
                                question_id = zop.get('id', '')
                                if question_id:
                                    link = f"https://www.zhihu.com/question/{question_id}"
                            except:
                                pass
                    
                    # 提取热度或排名
                    hot_elem = (item.select_one('.HotItem-metric') or 
                              item.select_one('.RankNumber') or 
                              item.select_one('.VoteCount') or
                              item.select_one('.HotItem-index'))
                    
                    heat_score = 0
                    if hot_elem:
                        hot_text = hot_elem.get_text().strip()
                        # 提取数字
                        hot_match = re.search(r'(\d+)', hot_text)
                        if hot_match:
                            heat_score = int(hot_match.group(1)) * 1000  # 转换为近似热度值
                    else:
                        # 根据排名估算热度
                        heat_score = max(1, 21 - i) * 40000  # 第1名约80万热度，第20名约4万
                    
                    # 提取描述或摘要
                    desc_elem = (item.select_one('.HotItem-excerpt') or 
                               item.select_one('.content') or
                               item.select_one('.excerpt'))
                    
                    description = ""
                    if desc_elem:
                        description = desc_elem.get_text().strip()
                    
                    trends.append({
                        "title": title,
                        "description": description,
                        "rank": i,
                        "heat_score": heat_score,
                        "url": link or f"https://www.zhihu.com/search?q={title}"
                    })
                except Exception as e:
                    app_logger.warning(f"解析单条知乎热榜失败: {e}")
                    continue
            
            # 如果还是没有数据，尝试直接解析页面中的JSON数据
            if not trends:
                # 查找页面中可能包含热榜数据的JSON
                json_pattern = r'\{"id":"\d+","url":"[^"]*","title":"[^"]*","excerpt":"[^"]*"[^\}]*\}'
                json_matches = re.findall(json_pattern, html)
                for i, json_str in enumerate(json_matches[:20], 1):
                    try:
                        data = json.loads(json_str)
                        title = data.get("title", "")
                        if title:
                            heat_score = max(1, 21 - i) * 40000  # 根据排名估算热度
                            
                            trends.append({
                                "title": title,
                                "description": data.get("excerpt", ""),
                                "rank": i,
                                "heat_score": heat_score,
                                "url": data.get("url", "") or f"https://www.zhihu.com/question/{data.get('id', '')}"
                            })
                    except Exception as e:
                        app_logger.warning(f"解析知乎页面JSON数据失败: {e}")
                        continue
            
            return trends
            
        except Exception as e:
            app_logger.error(f"解析知乎热榜HTML失败: {e}")
            return []
    
    async def _get_mock_data(self) -> List[Dict[str, Any]]:
        """获取模拟数据"""
        
        mock_trends = [
            {
                "title": "如何看待当前的人工智能发展趋势？",
                "description": "AI技术日新月异，各行各业都在积极拥抱人工智能...",
                "rank": 1,
                "heat_score": 850000,
                "url": "https://www.zhihu.com/question/12345678"
            },
            {
                "title": "程序员如何提升自己的核心竞争力？",
                "description": "在技术快速发展的时代，程序员需要不断学习和成长...",
                "rank": 2,
                "heat_score": 720000,
                "url": "https://www.zhihu.com/question/12345679"
            },
            {
                "title": "未来十年哪些技术会改变世界？",
                "description": "科技发展日新月异，哪些技术最有可能改变我们的生活...",
                "rank": 3,
                "heat_score": 650000,
                "url": "https://www.zhihu.com/question/12345680"
            }
        ]
        
        return mock_trends